import pymysql
import requests
import json
import io
import sys

# 今日头条热搜数据后端传输api，下面地址为JS请求的数据接口，f12从network/xhr中获取
start_url = 'https://i.snssdk.com/hot-event/hot-board/?origin=hot_board'
# 定义要插入数据的表名和数据库名
tablename='t_jinrihot'
databasename='resoudata'
# 定义浏览器标识和相应cookie，F12+network获取
# 添加请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36 Edg/96.0.1054.43',
    'cookie': 'gfsitesid=MjcxNTQwMTA3OHwxNjM4NjE5NjIyNTR8fDAGBgYGBgY; MONITOR_WEB_ID=45cb34d2-21eb-4e80-924e-089a6c5eb383; msToken=oDES286f2u1zI_uSm4ZHRfBgXSV5pC-j0SKG6SnMlKy7ul-Fjl4MwzuvM0TBUi4tWiWMGKVuJWXBtS509M5m39VCwjqzM9N7u-pgTnI1ZaA__9qcBxxGmROBPg==; gftoken=MjcxNTQwMTA3OHwxNjM4NjE5NjIyNTR8fDAGBgYGBgY'
}

# 改变标准输出的默认编码，防止spring boot运行时显示乱码
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8')

# 保存到数据库
def saveDAO(db,number,topic,hot,href):
    cursor = db.cursor()
    query = """insert into """+ tablename +""" (ranking,topic,hot,url,daydate,mindate) values (%s,%s,%s,%s,curdate(),curtime())"""
    cursor.execute(query,(number,topic,hot,href))
    cursor.close()

# 进行requests请求
def get(url):
    response = requests.get(url, headers=headers)
    if response. status_code == 200:
        print("今日头条热搜请求成功")
        parse(url)
    else:
        print("网页请求失败了")

# 将今日头条热搜的数据给爬下来
def parse(url):
    # 因为id自动递增，让自动增长从1开始，已经有了就跳过，让id连续
    db = pymysql.connect(host="localhost", user="root", password="root", database=databasename)
    cursor = db.cursor()
    str = """ALTER TABLE """+ tablename+""" AUTO_INCREMENT = 1"""
    cursor.execute(str)
    cursor.close()
    db.commit()

    # 由于是纯json数据，将其用文本形式保存
    response = requests.get(url, headers=headers)
    data = json.loads(response.text)
    # 获取json数据
    hotdata = data['data']

    number = 0
    inidata =[]
    try:
       for i in hotdata:
           topic = i['Title']
           hot = int(i['HotValue'])
           href = i['Url']
           data=[topic,hot,href]
           inidata.append(data)
       #今日头条排行不按热度，重新根据hot排序
       inidata.sort(key=lambda inidata : inidata[1])
       #反转列表，按热搜从大到小
       inidata=reversed(inidata)
       for i in inidata:
           number += 1
           saveDAO(db, number,i[0],i[1],i[2])
#            print(number,i[0],i[1],i[2])

    except BaseException as ex:
        print(ex)
    else:
        db.commit()
        db.close()
    response.close()

#程序从此处开始运行
if __name__ == "__main__":
    get(start_url)
